{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c08d617a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "# context.transpose(1, 2).contiguous().view\n",
    "# Tensor.contiguous(memory_format=torch.contiguous_format) → Tensor\n",
    "# Returns a contiguous in memory tensor containing the same data as self tensor. If self tensor is already in the specified memory format, this function returns the self tensor.\n",
    "\n",
    "# masked_fill_\n",
    "\n",
    "# seq_k.data.eq(0).unsqueeze(1)\n",
    "# seq_len = x.size(1)\n",
    "\n",
    "\n",
    "# torch.matmul\n",
    "# nn.LayerNorm(d_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "01ce2b49",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0, 1]) torch.Size([2])\n",
      "tensor([[0, 1]]) torch.Size([1, 2])\n"
     ]
    }
   ],
   "source": [
    "# 1. 初始化\n",
    "x = torch.tensor(2, 5)\n",
    "x = torch.tensor([1, 2, 3])\n",
    "x = torch.randn(2, 3)\n",
    "pos = torch.arange(2, dtype=torch.long)\n",
    "\n",
    "print(pos,pos.shape)\n",
    "pos = pos.unsqueeze(0)\n",
    "print(pos,pos.shape)\n",
    "# print(pos.expand_as(x) )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "c3a5ff9a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1, 2, 3])\n",
      "tensor([[1, 2, 3, 1, 2, 3],\n",
      "        [1, 2, 3, 1, 2, 3],\n",
      "        [1, 2, 3, 1, 2, 3],\n",
      "        [1, 2, 3, 1, 2, 3]])\n"
     ]
    }
   ],
   "source": [
    "# 2.增、删、改 tensor元素\n",
    "x = torch.tensor([1, 2, 3])\n",
    "print(x)\n",
    "x = x.repeat(4, 2)\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "557ae282",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3. 升降维,转置，打平，"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54a93fef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4. 索引"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "f56fce7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-1000000000,           2,           3],\n",
       "        [          4, -1000000000,           6]])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = torch.tensor([[1,0,0],[0,1,0]])\n",
    "x = torch.tensor([[1,2,3],[4,5,6]])\n",
    "x.masked_fill_(m, -1e9)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "02d7125e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4])\n",
      "tensor([[1, 2, 3, 4]])\n",
      "tensor([[1],\n",
      "        [2],\n",
      "        [3],\n",
      "        [4]])\n",
      "torch.Size([1, 4])\n",
      "torch.Size([4, 1])\n",
      "~~~~~\n",
      "torch.Size([4, 1])\n",
      "torch.Size([1, 4, 1])\n",
      "torch.Size([4, 1, 1])\n",
      "torch.Size([4, 1, 1])\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([1, 2, 3, 4])\n",
    "print(x.shape)\n",
    "print(torch.unsqueeze(x, 0)) \n",
    "print(torch.unsqueeze(x, 1)) \n",
    "print(torch.unsqueeze(x, 0).shape) \n",
    "print(torch.unsqueeze(x, 1).shape) \n",
    "\n",
    "# A dim value within the range [-input.dim() - 1, input.dim() + 1) can be used. \n",
    "# Negative dim will correspond to unsqueeze() applied at dim = dim + input.dim() + 1.\n",
    "                              \n",
    "x = torch.tensor([[1],\n",
    "        [2],\n",
    "        [3],\n",
    "        [4]])\n",
    "print(\"~~~~~\")\n",
    "print(x.shape)\n",
    "print(torch.unsqueeze(x, 0).shape)\n",
    "print(torch.unsqueeze(x, 1).shape)\n",
    "print(torch.unsqueeze(x, -2).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6ec99dc8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 1])\n",
      "tensor([[1, 1, 1, 1],\n",
      "        [2, 2, 2, 2],\n",
      "        [3, 3, 3, 3]])\n",
      "tensor([[1, 1, 1, 1],\n",
      "        [2, 2, 2, 2],\n",
      "        [3, 3, 3, 3]])\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([[1], [2], [3]])\n",
    "print(x.size())\n",
    "\n",
    "print(x.expand(3, 4)) #3行4列\n",
    "print(x.expand(-1, 4))   # -1 means not\n",
    "\n",
    "# x.expand_as() 等同expand"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2939557e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 1],\n",
       "        [4, 3]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = torch.tensor([[1, 2], [3, 4]])\n",
    "torch.gather(t, 1, torch.tensor([[0, 0], [1, 0]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "c80159e4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 2.0502,  0.9128,  0.3904, -1.0539, -0.3555,  0.3123,  1.3060, -0.4311,\n",
      "        -0.3213,  1.1517, -1.5596,  0.7091, -1.6580, -0.0346, -0.5737,  0.6815]) torch.Size([16])\n"
     ]
    }
   ],
   "source": [
    "x = torch.randn(4, 4)\n",
    "y = x.view(16)\n",
    "print(y,y.size())\n",
    "# print(x.transpose(1, 2) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "a5779b41",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.2202,  1.9603, -0.9585],\n",
      "        [-0.2536,  0.0324, -1.6509]]) torch.Size([2, 3])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.2202, -0.2536],\n",
       "        [ 1.9603,  0.0324],\n",
       "        [-0.9585, -1.6509]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.randn(2, 3)\n",
    "print(x,x.size())\n",
    "torch.transpose(x, 0, 1)\n",
    "\n",
    "# transpose, 负数意思是从后往前数"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af534f26",
   "metadata": {},
   "source": [
    "![矩阵点乘](images/dot_product.webp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "191adf09",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 2]) torch.Size([2, 1])\n",
      "tensor([[17],\n",
      "        [39]])\n",
      "torch.Size([2, 2]) torch.Size([2, 1]) torch.Size([2, 1])\n",
      "tensor([[17],\n",
      "        [39]])\n",
      "tensor([[ 5, 10],\n",
      "        [18, 24]])\n"
     ]
    }
   ],
   "source": [
    "tensor1 = torch.tensor([[1,2],[3,4]])\n",
    "tensor2 = torch.tensor([[5],[6]])\n",
    "print(tensor1.shape,tensor2.shape)\n",
    "\n",
    "#维度变化：[a,b] * [b,c] = [a,c]\n",
    "ret = torch.matmul(tensor1, tensor2)\n",
    "ret1 = tensor1.matmul(tensor2)\n",
    "print(ret1) \n",
    "\n",
    "ret2 = tensor1 * tensor2 #注意差别\n",
    "print(tensor1.shape,tensor2.shape,ret.shape)\n",
    "print(ret)\n",
    "print(ret2)\n",
    "\n",
    "# https://pytorch.org/docs/master/generated/torch.matmul.html?highlight=matmul#torch.matmul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a804d197",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://pytorch.org/docs/master/generated/torch.gather.html?highlight=gather#torch.gather\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
